'''
Created on 2020年1月30日
获取所有股票数据
@author: JM
'''
import pandas as pd
import time
from app_config import get_pro, get_engine_ts

pro = get_pro()
engine_ts = get_engine_ts()


def read_data():
    sql = """SELECT * FROM dividend"""
    df = pd.read_sql_query(sql, engine_ts)
    return df


def write_data(df_):
    res = df_.to_sql('dividend', engine_ts, index=False, if_exists='append', chunksize=5000)
    print(res)


def get_data(_ex_date):
    df = pro.dividend(ex_date=_ex_date,
                      fields='ts_code,end_date,ann_date,div_proc,stk_div,stk_bo_rate,stk_co_rate,cash_div,'
                             'cash_div_tax,record_date,ex_date,pay_date,div_listdate,imp_ann_date,base_date,base_share')

    df['id'] = df['ts_code'].astype(str) + "_" + df['div_proc'].astype(str) + "_" + df['end_date'].astype(str) + "_" + \
               df['pay_date'].astype(str)
    return df


def get_daily(ex_date):
    for _ in range(10):
        try:
            df = get_data(ex_date)
            write_data(df)
        except Exception as e:
            print(f"An unexpected error occurred: {e}")
            time.sleep(10)
        else:
            return
    print("error error error error error" + ex_date)


def fetch_data(start_date, end_date):
    df_q = pro.trade_cal(exchange='SSE', is_open='1',
                         start_date=start_date,
                         end_date=end_date,
                         fields='cal_date')

    df_db = pd.read_sql_query("select DISTINCT(ex_date) as cal_date from dividend", engine_ts)
    # 找出df_q中"cal_date"列的数据, 这些数据不在df_db的"cal_date"列中
    result = df_q[~df_q['cal_date'].isin(df_db['cal_date'])]

    result_len = len(result)
    is_do = 0

    for index, row in result.iterrows():  #
        is_do = is_do + 1
        # 每天的交易日期
        date = row["cal_date"]
        print("每天交易日期" + str(date))
        print("all：" + str(result_len) + " remain: " + str(result_len - is_do))
        get_daily(date)


if __name__ == '__main__':
    fetch_data('20180601', '20251201')
